package com.zll.jsoupdemo.reptile;

import com.zll.jsoupdemo.dao.MzImgDao;
import com.zll.jsoupdemo.entity.MzImg;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
 * 爬取妹子图片
 * @author zll
 */
@Component
@Slf4j
public class ReptileMeiZi implements ApplicationRunner {

    @Autowired
    private RestTemplate restTemplate;

    @Autowired
    private MzImgDao mzImgDao;

    @Override
    public void run(ApplicationArguments args) throws Exception {
        List<MzImg> mzImgs = new ArrayList<>();
        Set<Integer> errorPageNum = new HashSet<>();
        String basePath = "F:" + File.separator + "meizitu" + File.separator;
        String refererPageInit = "https://www.mzitu.com/";
        for (int i = 1; i <= 241; i++) {
            parseHtml( i, mzImgs, errorPageNum,refererPageInit, basePath);
            refererPageInit = "https://www.mzitu.com/page/" + i + "/";
        }
        if (CollectionUtils.isNotEmpty(errorPageNum)) {
            log.error("最后还有{}页执行失败", StringUtils.join(errorPageNum,","));
        }
        mzImgDao.insertBatch(mzImgs);
    }

    /**
     * 请求html、解析并保存磁盘
     * @param i 页数
     * @param mzImgs 妹子对象集合
     * @param errorPageNum 错误集合
     * @param refererPageInit 请求头Referer值
     * @param basePath 保存到磁盘的基础目录
     */
    private void parseHtml(Integer i,List<MzImg> mzImgs,Set<Integer> errorPageNum,String refererPageInit,String basePath) {
        String url = "https://www.mzitu.com/page/" + i + "/";
        Connection connect = Jsoup.connect(url);
        connect.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        connect.header("Accept-Encoding", "gzip, deflate, sdch");
        connect.header("Accept-Language", "zh-CN,zh;q=0.8");
        connect.header("Sec-Fetch-Dest", "document");
        connect.header("Upgrade-Insecure-Requests", "1");
        connect.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
        connect.header("Referer", refererPageInit);
        Document document = null;
        try {
            document = connect.get();
        } catch (IOException e) {
            log.error("获取第{}页失败",i);
            errorPageNum.add(i);
            return;
        }
        Elements imgs = document.getElementsByTag("img");
        for (Element img : imgs) {
            String src = img.attr("data-original");
            String alt = img.attr("alt");
            log.info("第{}页:alt->{};src->{}", i, alt, src);
            MzImg mzImg = new MzImg();
            mzImg.setAlt(alt);
            mzImg.setSrc(src);
            mzImgs.add(mzImg);
            if (StringUtils.isNotBlank(src)) {
                HttpHeaders httpHeaders = new HttpHeaders();
                httpHeaders.set("Referer",refererPageInit);
                httpHeaders.set("Sec-Fetch-Dest","image");
                httpHeaders.set("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
                HttpEntity<Resource> httpEntity = new HttpEntity<>(httpHeaders);
                ResponseEntity<byte[]> responseEntity = restTemplate.exchange(src, HttpMethod.GET, httpEntity, byte[].class);
                String newName = alt + "." + StringUtils.substringAfterLast(src, ".");
                File file = new File(basePath+newName);
                try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
                    fileOutputStream.write(responseEntity.getBody());
                } catch (Exception e) {
                    log.error("保存文件:{}出错",newName);
                }
            }

        }

    }
}
